from urllib.parse import urlencode

from lxml.etree import XMLSyntaxError
from requests.exceptions import ConnectionError
import requests
from pyquery import PyQuery as pq
from config import *
import pymongo



client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
base_url ='http://weixin.sogou.com/weixin?'

headers = {

'Cookie': 'SUV=1510117598923983; SMYUV=1510117598924776; dt_ssuid=1785442896; pex=C864C03270DED3DD8A06887A372DA219231FFAC25A9D64AE09E82AED12E416AC; ssuid=2219378133; CXID=4C3C881204E3E1BFEE77E11E25AE27B5; GOTO=; _ga=GA1.2.535693272.1513856981; IPLOC=CN5301; pgv_pvi=4670192640; tv_play_records=tvshow_2088721:20180323; LSTMV=267%2C584; LCLKINT=17782; sw_uuid=2087855278; sg_uuid=9220209661; SUID=C2678DB63220910A000000005A2263E4; ad=PZllllllll2bSN$mlllllVHyPI9lllllT$hpKyllll9lllll9Zlll5@@@@@@@@@@; ABTEST=5|1532324872|v1; SNUID=F7CDCCCF111561B53FBD7658112AB9F0; weixinIndexVisited=1; ppinf=5|1532324885|1533534485|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToyNzolRTYlOUMlQjElRTUlQkUlQjclRTklOTMlQUR8Y3J0OjEwOjE1MzIzMjQ4ODV8cmVmbmljazoyNzolRTYlOUMlQjElRTUlQkUlQjclRTklOTMlQUR8dXNlcmlkOjQ0Om85dDJsdU54T0xvLXd4NEVWUFB1UVhhaGRVUGNAd2VpeGluLnNvaHUuY29tfA; pprdig=CfmcNWAx_IJhkk9o4vvd59yiDEeSIHZO1wl_MaPdFff4VezGchR64MpmQ-YYUWHJysuifPyXy3frV4c4WvlLFEVXwEeVy_cP9nnm0MW4cHbg5V6hUg156ZAz6nzFzZ48I2zPocBdsz54Uu3L2z0_BU5GG_JZxEdniCeRX2e9mY8; sgid=03-34103589-AVtVbBWsaSJIeRMbXsaQfyQ; ppmdig=15323248860000006134411c2bce09765b8f5f8b4e8041b2; sct=6; JSESSIONID=aaaXrAT-7Lj8SaKxslHsw',
'Host': 'weixin.sogou.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}


proxy = None
max_count = 5
def get_proxy():
    try:
        response = requests.get(proxy_pool_url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None

def get_html(url, count=1):
    print('crawling', url)
    print('trying count', count)
    global proxy
    if count >= max_count:
        print('tried too many countS')
        return None
    try:
        if proxy:
            proxies = {
                'http': 'http://' + proxy
            }
            response = requests.get(url, allow_redirects=False, headers=headers, proxies=proxies)
            response.encoding = 'utf-8'
        else:
            response = requests.get(url, allow_redirects=False, headers=headers)
            response.encoding = 'utf-8'
        if response.status_code == 200:
            return response.text
        if response.status_code == 302:
            # Need proxy
            print(302)
            proxy = get_proxy()
            if proxy:
                print('using proxy', proxy)
                return get_html(url)
            else:
                print('get proxy failed')
                return None
    except ConnectionError as e:
        print('error 0ccurred', e.args)
        proxy = get_proxy()
        count += 1
        return get_html(url, count)




def get_index(keyword, page):
    data = {
        'query': keyword,
        'type': 2,
        'page': page


    }
    querise = urlencode(data)
    url = base_url + querise
    html = get_html(url)
    return html

def parse_index(html):

    doc = pq(html)
    items = doc('.news-box .news-list li .txt-box h3 a').items()
    for item in items:
        yield item.attr('href')

def get_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None

def parse_detail(html):
    try:
        doc = pq(html)
        title = doc('.rich_media_title').text()
        content = doc('.rich_media_content').text()
        date = doc('#publish_time').text()
        nickname = doc('#js_profile_qrcode > div > strong').text()
        wechat = doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
        return {
            'title': title,
            'content': content,
            'data': date,
            'nickname': nickname,
            'wechat': wechat

        }
    except XMLSyntaxError:
        return None

def mongodb(data):
    if db[MONGO_TABLE].update({'title': data['title']}, {'$set': data}, True):
        print('saved to mongo', data['title'])
    else:
        print('saved to mongo failed', data['title'])

def main():
    for page in range(1, 101):
        html = get_index(keyword, page)
        if html:
            ar_urls = parse_index(html)
            for ar_url in ar_urls:
                ar_html = get_detail(ar_url)
                if ar_html:
                    ar_data = parse_detail(ar_html)
                    print(ar_data)
                    if ar_data:
                        mongodb(ar_data)

if __name__ == '__main__':
    main()